import requests
import os
import sys
import time
# from func_def import list_file

from contextlib import closing
from util.log_handler import LogHandler
from util.web_request import WebRequest

from util.log_handler import LogHandler
from util.util_function import DownloadFile

# 获取文件名
def get_filename(path):
	filename = os.path.basename(path)
	filename = filename.split('.')[0]
	return filename

# 获取当前执行程序的文件名
def get_execute_filename():
	return get_filename(__file__)

log = LogHandler(get_execute_filename())

# 遍历文件
def list_file(path, ext='.jpg'):
    L=[]
    for root, dirs, files in os.walk(path):
        for file in files:
            file_ext = file.endswith(ext)
            if file_ext:
                fpath = os.path.join(root, file)
                L.append(fpath)
    return L

# 逐行读取文件
def readline_file(path):
    # print(path)
    f = open(path, 'r')
    urls = f.readlines()
    f.close()
    return urls

# 判断文件夹是否存在，不存在则创建
def exist_dir(dir_path):
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)

'''/home/ubuntu/workspace/pbs_old/app/cloud_chart/image_url/2016/3DS/01.txt'''

files = list_file('spider/cloud_chart/image_url', '.txt')

if __name__ == '__main__':
    base_dir_path = '/media/ubuntu/08f29263-3882-43db-96fd-febb6f2e2e5f/cloud_chart'
    # exist_dir(base_dir_path)
    for filename in files:
        # 需要保存文件夹的相对路径
        dir_name = os.path.join(filename.split('/')[-3], filename.split('/')[-2])
        # 需要保存文件夹的绝对路径
        dir_path = os.path.join(base_dir_path, dir_name)
        exist_dir(dir_path)
        urls = readline_file(filename)
        # time.sleep(10)
        for url in urls:
            url = url[:-1]
            # img_name = url[-34:]
            img_name = url.split('=')[-1]
            # 调用下载图片方法
            DownloadFile(url, dir_path, img_name)